
from sklearn.datasets import fetch_openml
from matplotlib import pyplot as plt
import keras
from threading import Thread

import seaborn as sns; 
from sklearn.metrics import confusion_matrix, classification_report, accuracy_score

from sklearn.metrics import accuracy_score
import matplotlib.pyplot as plt
from keras.models import Sequential
from keras.layers import Dense
from tensorflow.keras.utils import to_categorical
import numpy as np
import pandas as pd
import matplotlib.ticker as ticker


data = fetch_openml('Fashion-MNIST', as_frame=False, parser='auto')


list(data)

['data',
 'target',
 'frame',
 'categories',
 'feature_names',
 'target_names',
 'DESCR',
 'details',
 'url']


print(data.DESCR)

**Author**: Han Xiao, Kashif Rasul, Roland Vollgraf  
**Source**: [Zalando Research](https://github.com/zalandoresearch/fashion-mnist)  
**Please cite**: Han Xiao and Kashif Rasul and Roland Vollgraf, Fashion-MNIST: a Novel Image Dataset for Benchmarking Machine Learning Algorithms, arXiv, cs.LG/1708.07747  

Fashion-MNIST is a dataset of Zalando's article images, consisting of a training set of 60,000 examples and a test set of 10,000 examples. Each example is a 28x28 grayscale image, associated with a label from 10 classes. Fashion-MNIST is intended to serve as a direct drop-in replacement for the original MNIST dataset for benchmarking machine learning algorithms. It shares the same image size and structure of training and testing splits. 

Raw data available at: https://github.com/zalandoresearch/fashion-mnist

### Target classes
Each training and test example is assigned to one of the following labels:
Label  Description  
0  T-shirt/top  
1  Trouser  
2  Pullover  
3  Dress  
4  Coat  
5  Sandal  
6  Shirt  
7  Sneaker  
8  Bag  
9  Ankle boot

Downloaded from openml.org.


data.data.shape

(70000, 784)


data.target.shape

(70000,)


X = data.data
y = to_categorical(data.target)


def plot_image(image_data):
    image = image_data.reshape(28, 28)
    plt.imshow(image, cmap="binary")
    plt.axis("off")


plt.figure(figsize=(9, 9))
for idx, image_data in enumerate(X[:100]):
    plt.subplot(10, 10, idx + 1)
    plot_image(image_data)
plt.subplots_adjust(wspace=0, hspace=0)
plt.show()


X_train, X_test, y_train, y_test = X[:60000], X[60000:], y[:60000], y[60000:]

X_train = X_train.reshape((60000, 28 * 28))  # flattening the 28x28 image into dim=1 (784,) vector (and over all images in the training set)
X_test = X_test.reshape((10000, 28 * 28))

X_train = X_train.astype("float32") / 255  # normalizing the data to the range [0,1] by min-max range, and casting it to float32
X_test = X_test.astype("float32") / 255


def fit_model(model, epochs, batch_size, verbose):
    model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'])
    history = model.fit(X_train, y_train, epochs=epochs, batch_size=batch_size, verbose=verbose)
    return model, history


def one_hot_to_label(one_hot):
    label = list()

    for i in range(len(one_hot)):
        label.append(np.argmax(one_hot[i]))

    return label


def evaluate_model(y_test, y_pred, labels):
  y_pred = one_hot_to_label(y_pred)
  y_test = one_hot_to_label(y_test)
  cm = confusion_matrix(y_test, y_pred)
  cr = classification_report(y_test, y_pred, zero_division=0)

  
  sns.heatmap(cm.T, square=True, annot=True, fmt='d', xticklabels = labels, yticklabels = labels)
  plt.xlabel('true label')
  plt.ylabel('predicted label')
  print(cr)
  plt.show()
  return cm, cr


def evaluate_histories(histories, models, figsize):
    _,(ax1,ax2)=plt.subplots(2,figsize=figsize)

    ax1.xaxis.set_major_locator(ticker.MaxNLocator(integer=True))
    ax2.xaxis.set_major_locator(ticker.MaxNLocator(integer=True))

    for i in range(len(histories)):
        val_acc = histories[i].history['accuracy']
        val_loss = histories[i].history['loss']
        ax1.plot(val_acc, label=models[i])
        ax2.plot(val_loss, label=models[i])
    ax1.set_ylabel('Accuracy')
    ax2.set_ylabel('Loss')
    ax1.set_xlabel('Epochs')
    ax2.set_xlabel('Epochs')
    ax1.legend()
    ax2.legend()
    plt.show()


def classification_report_to_df(cr):
    cr_data = [x.split() for x in cr.split('\n')]
    cr_df = pd.DataFrame(columns=cr_data[0][:-1])
    for data in cr_data[2:12]:
        data_df = pd.DataFrame([data[1:-1]], columns=cr_data[0][:-1])   
        cr_df = pd.concat([cr_df, data_df], ignore_index=True)

    return(cr_df)


def evaluate_classification_reports(cr_array):
    df = pd.DataFrame()

    for i in range(len(cr_array)):
        cr_df = classification_report_to_df(cr_array[i])[3:]
        cr_df.index = cr_df.index + 1
        cr_df = cr_df.stack()
        cr_df.index = cr_df.index.map('{0[1]}_{0[0]}'.format)
        cr_df = cr_df.to_frame().T
        df = pd.concat([df, cr_df], ignore_index=True)

    object_columns = df.columns[df.dtypes.eq(object)]
    df[object_columns] = df[object_columns].apply(
        pd.to_numeric,
        errors='coerce',
        axis=0
    )

    df_min = pd.DataFrame(df).T
    df_max = pd.DataFrame(df.idxmax()).value_counts()

    best_model = df_max.T.idxmax()[0]
    stable_model = df_min.min().idxmax()

    return best_model, stable_model


def make_model(layers, input, output):
    model = Sequential()

    if len(layers) == 0:
        model.add(Dense(output, input_dim=input, activation='softmax'))

    else:
        model.add(Dense(layers[-1], input_dim=input, activation='relu'))
        
        for i in range(len(layers) - 1):
            model.add(Dense(i, activation='relu'))

        model.add(Dense(10, activation='softmax'))

    return model


def build_and_evaluate_model(epochs, batch_size, input, output, verbose, layers, result, index):
    model = make_model(layers, input, output)
    model, history = fit_model(model,epochs, batch_size, verbose)
    y_pred = model.predict(X_test, verbose=0)
    # with results_lock:
    result[index] = {'history': history, 'y_test': y_test, 'y_pred': y_pred}
        # finished_threads += 1  # Increment the counter for finished threads
        # print("Threads done:", finished_threads)
    print('Done:', layers)


def build_and_evaluate_models(models):
    
    labels =  ["T-shirt/top", "Trouser", "Pullover", "Dress", "Coat", "Sandal", "Shirt", "Sneaker", "Bag", "Ankle boot"]
    epochs =  5
    batch_size =  128
    input =  784
    output =  10
    verbose =  0


    model_results = {
        'models': models,
        'history_array':[],
        'cm_array':[],
        'cr_array':[]
    }

    threads = [None] * len(models)
    results = [None] * len(models)

    for i in range(len(threads)):
        threads[i] = Thread(target=build_and_evaluate_model, args=(epochs, batch_size, input, output, verbose, models[i], results, i))
        threads[i].start()

    for i in range(len(threads)):
        threads[i].join()

    for i in range(len(results)):
        print(models[i])
        cm, cr = evaluate_model(y_test, results[i]['y_pred'], labels)

        model_results['history_array'].append(results[i]['history'])
        model_results['cm_array'].append(cm)
        model_results['cr_array'].append(cr)
        
    evaluate_histories(model_results['history_array'], models, (8, 100))
    return evaluate_classification_reports(model_results['cr_array'])


nodes = np.arange(10, 101, 10)

models = [[]]

for first_layer in nodes:
    for second_layer in nodes:
        models.append([first_layer, second_layer])

best_model, stable_model = build_and_evaluate_models(models)
print('Best model: ', models[best_model])
print('Stable model: ', models[stable_model])

Done: []
Done: [30, 10]
Done: [30, 30]
Done: [60, 40]
Done: [30, 20]
Done: [40, 10]
Done: [50, 10]
Done: [80, 90]
Done: [20, 10]
Done: [90, 20]
Done: [70, 20]
Done: [20, 30]
Done: [80, 10]
Done: [100, 30]
Done: [40, 20]
Done: [20, 40]
Done: [40, 80]
Done: [40, 40]
Done: [50, 20]
Done: [100, 20]
Done: [40, 30]
Done: [50, 60]
Done: [60, 30]
Done: [10, 30]
Done: [20, 20]
Done: [100, 10]
Done: [10, 20]
Done: [30, 80]
Done: [80, 30]
Done: [10, 10]
Done: [80, 20]
Done: [60, 10]
Done: [50, 30]
Done: [20, 90]
Done: [60, 20]
Done: [90, 10]
Done: [70, 30]
Done: [20, 50]
Done: [90, 30]
Done: [10, 90]
Done: [100, 40]
Done: [90, 90]
Done: [20, 60]
Done: [70, 10]
Done: [10, 50]
Done: [60, 50]
Done:Done: [10, 40]
 [40, 70]
Done: [90, 40]
Done: [10, 60]
Done: [50, 50]
Done: [30, 40]
Done: [50, 70]
Done: [30, 50]
Done: [100, 70]
Done: [50, 40]
Done: [80, 70]
Done: [90, 50]
Done: [100, 50]
Done: [100, 90]
Done: [90, 70]
Done: [10, 70]
Done: [20, 70]
Done: [80, 40]
Done: [70, 40]
Done: [70, 60]
Done: [30, 60]
Done: [60, 80]
Done: [90, 60]
Done: [40, 60]
Done: [90, 80]
Done: [30, 90]
Done: [70, 80]
Done:Done: [20, 80]
 [50, 90]
Done: [30, 70]
Done: [70, 100]
Done: [60, 60]
Done: [40, 50]
Done: [100, 60]
Done: [30, 100]
Done: [20, 100]
Done: [10, 100]
Done: [80, 80]
Done: [10, 80]
Done: [40, 90]
Done: [80, 100]
Done: [70, 90]
Done: [70, 50]
Done: [100, 80]
Done: [70, 70]
Done: [90, 100]
Done: [80, 60]
Done: [60, 70]
Done: [50, 80]
Done: [80, 50]
Done: [40, 100]
Done: [60, 90]
Done: [100, 100]
Done: [60, 100]
Done: [50, 100]
[]
              precision    recall  f1-score   support

           0       0.79      0.82      0.80      1000
           1       0.97      0.95      0.96      1000
           2       0.76      0.68      0.72      1000
           3       0.86      0.84      0.85      1000
           4       0.71      0.79      0.75      1000
           5       0.95      0.90      0.92      1000
           6       0.61      0.57      0.59      1000
           7       0.90      0.94      0.92      1000
           8       0.90      0.97      0.93      1000
           9       0.93      0.94      0.94      1000

    accuracy                           0.84     10000
   macro avg       0.84      0.84      0.84     10000
weighted avg       0.84      0.84      0.84     10000

[10, 10]
              precision    recall  f1-score   support

           0       0.10      1.00      0.18      1000
           1       0.00      0.00      0.00      1000
           2       0.00      0.00      0.00      1000
           3       0.00      0.00      0.00      1000
           4       0.00      0.00      0.00      1000
           5       0.00      0.00      0.00      1000
           6       0.00      0.00      0.00      1000
           7       0.00      0.00      0.00      1000
           8       0.00      0.00      0.00      1000
           9       0.00      0.00      0.00      1000

    accuracy                           0.10     10000
   macro avg       0.01      0.10      0.02     10000
weighted avg       0.01      0.10      0.02     10000

[10, 20]
              precision    recall  f1-score   support

           0       0.10      1.00      0.18      1000
           1       0.00      0.00      0.00      1000
           2       0.00      0.00      0.00      1000
           3       0.00      0.00      0.00      1000
           4       0.00      0.00      0.00      1000
           5       0.00      0.00      0.00      1000
           6       0.00      0.00      0.00      1000
           7       0.00      0.00      0.00      1000
           8       0.00      0.00      0.00      1000
           9       0.00      0.00      0.00      1000

    accuracy                           0.10     10000
   macro avg       0.01      0.10      0.02     10000
weighted avg       0.01      0.10      0.02     10000

[10, 30]
              precision    recall  f1-score   support

           0       0.10      1.00      0.18      1000
           1       0.00      0.00      0.00      1000
           2       0.00      0.00      0.00      1000
           3       0.00      0.00      0.00      1000
           4       0.00      0.00      0.00      1000
           5       0.00      0.00      0.00      1000
           6       0.00      0.00      0.00      1000
           7       0.00      0.00      0.00      1000
           8       0.00      0.00      0.00      1000
           9       0.00      0.00      0.00      1000

    accuracy                           0.10     10000
   macro avg       0.01      0.10      0.02     10000
weighted avg       0.01      0.10      0.02     10000

[10, 40]
              precision    recall  f1-score   support

           0       0.10      1.00      0.18      1000
           1       0.00      0.00      0.00      1000
           2       0.00      0.00      0.00      1000
           3       0.00      0.00      0.00      1000
           4       0.00      0.00      0.00      1000
           5       0.00      0.00      0.00      1000
           6       0.00      0.00      0.00      1000
           7       0.00      0.00      0.00      1000
           8       0.00      0.00      0.00      1000
           9       0.00      0.00      0.00      1000

    accuracy                           0.10     10000
   macro avg       0.01      0.10      0.02     10000
weighted avg       0.01      0.10      0.02     10000

[10, 50]
              precision    recall  f1-score   support

           0       0.10      1.00      0.18      1000
           1       0.00      0.00      0.00      1000
           2       0.00      0.00      0.00      1000
           3       0.00      0.00      0.00      1000
           4       0.00      0.00      0.00      1000
           5       0.00      0.00      0.00      1000
           6       0.00      0.00      0.00      1000
           7       0.00      0.00      0.00      1000
           8       0.00      0.00      0.00      1000
           9       0.00      0.00      0.00      1000

    accuracy                           0.10     10000
   macro avg       0.01      0.10      0.02     10000
weighted avg       0.01      0.10      0.02     10000

[10, 60]
              precision    recall  f1-score   support

           0       0.10      1.00      0.18      1000
           1       0.00      0.00      0.00      1000
           2       0.00      0.00      0.00      1000
           3       0.00      0.00      0.00      1000
           4       0.00      0.00      0.00      1000
           5       0.00      0.00      0.00      1000
           6       0.00      0.00      0.00      1000
           7       0.00      0.00      0.00      1000
           8       0.00      0.00      0.00      1000
           9       0.00      0.00      0.00      1000

    accuracy                           0.10     10000
   macro avg       0.01      0.10      0.02     10000
weighted avg       0.01      0.10      0.02     10000

[10, 70]
              precision    recall  f1-score   support

           0       0.10      1.00      0.18      1000
           1       0.00      0.00      0.00      1000
           2       0.00      0.00      0.00      1000
           3       0.00      0.00      0.00      1000
           4       0.00      0.00      0.00      1000
           5       0.00      0.00      0.00      1000
           6       0.00      0.00      0.00      1000
           7       0.00      0.00      0.00      1000
           8       0.00      0.00      0.00      1000
           9       0.00      0.00      0.00      1000

    accuracy                           0.10     10000
   macro avg       0.01      0.10      0.02     10000
weighted avg       0.01      0.10      0.02     10000

[10, 80]
              precision    recall  f1-score   support

           0       0.10      1.00      0.18      1000
           1       0.00      0.00      0.00      1000
           2       0.00      0.00      0.00      1000
           3       0.00      0.00      0.00      1000
           4       0.00      0.00      0.00      1000
           5       0.00      0.00      0.00      1000
           6       0.00      0.00      0.00      1000
           7       0.00      0.00      0.00      1000
           8       0.00      0.00      0.00      1000
           9       0.00      0.00      0.00      1000

    accuracy                           0.10     10000
   macro avg       0.01      0.10      0.02     10000
weighted avg       0.01      0.10      0.02     10000

[10, 90]
              precision    recall  f1-score   support

           0       0.10      1.00      0.18      1000
           1       0.00      0.00      0.00      1000
           2       0.00      0.00      0.00      1000
           3       0.00      0.00      0.00      1000
           4       0.00      0.00      0.00      1000
           5       0.00      0.00      0.00      1000
           6       0.00      0.00      0.00      1000
           7       0.00      0.00      0.00      1000
           8       0.00      0.00      0.00      1000
           9       0.00      0.00      0.00      1000

    accuracy                           0.10     10000
   macro avg       0.01      0.10      0.02     10000
weighted avg       0.01      0.10      0.02     10000


# models = np.reshape(np.arange(10, 1000, 10), (-1, 1))

# best_model, stable_model = build_and_evaluate_models(models)
# print('Best model: ', models[best_model])
# print('Stable model: ', models[stable_model])


# models = np.reshape(np.arange(10, 1000, 10), (-1, 1))

# best_model, stable_model = build_and_evaluate_models(models)
# print('Best model: ', models[best_model])
# print('Stable model: ', models[stable_model])


# models = np.reshape(np.arange(10, 50, 1), (-1, 1))

# best_model, stable_model = build_and_evaluate_models([[], [10], [20]])
# print('Best model: ', models[best_model])
# print('Stable model: ', models[stable_model])

1. Using Keras Datasets, obtain the fMNIST data set. Explore and describe it as you would do with any new dataset that you start working with (2 points).¶

2. Train a dense neural network using Keras¶